package FeedbackedQuery;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;


public class Client {
	
	
	/**
	 * Definition of Error Codes
	 */
	private static int _ERROR_NUMBEROF_QUERY_WORDS_0_ = -1;
	private static int _ERROR_ILLEGLE_PRECISION_VALUE = -2;
	private static int _ERROR_ARGUMENT_FORMAT_ = -3;
	private static int _ERROR_MALIC_URL_FORMAT_ = -4;
	private static int _ERROR_IN_INTERNET_IO_ = -5;
	private static int _ERROR_IN_PARSE_XML_RESULT_ = -6;
	
	
	private static HashSet<String> stopWords;
	
	/**
	 * The coefficient for query expansion 
	 */
	private double alpha = 1.0;
	private double beta = 0.75;
	private double gama = -0.15;
	
	
	/**
	 * The static element for reference
	 */
	private static Client client;		

	/**
	 * The arguments from command line
	 */
	private String[] argz;
	
	
	/**
	 * Elements parsed from arguments
	 */
	ArrayList<String> keywords;

	
	/**
	 * The query vector
	 */
	double[] vector;
	double precision;
	String yahooID;

	static final String yahooURL_1 = "http://boss.yahooapis.com/ysearch/web/v1/";
	static final String yahooURL_2 = "?appid=";
	static final String yahooURL_3 = "&format=xml&abstract=long";

	
	/**
	 * Elements for accessing the web to get the articles information
	 */
	URL url;	
	URLConnection connection;
	InputStream inStream;
	

	/**
	 * Element for outputting the results to a transcript
	 */
	File file;
	FileWriter out = null;
	BufferedWriter bw = null;
	
		
	/**
	 * Set of Relevant and irrelevant articles
	 */
	HashSet<Article> relevantArticles;
	HashSet<Article> irrelevantArticles;

	
	
	/**
	 * The array of newly parsed articles;
	 */
	ArrayList<Article> newArticles;
	
	
	/**
	 * Recording the id of latest discovered word, starting from 0
	 * it's value + 1 is the number of all words, or #dimensions or 
	 * all vectors.
	 */
	int currentWordId;

	
	/**
	 * Map from word to its ID
	 */
	HashMap<String, Integer> wordIdMap;
	ArrayList<String> IdWordMap;	// And from Id to the word

			
	/**
	 * Map from word to the set of articles containing it
	 */
	SetMap<String, Article>wordToArticlemap;
	
	
	
		
	/**
	 *  Assistant function, because our argz received are of the form:
	 * 'query words' precision appID, this function parses them 
	 * @param argz
	 * @return 0 if no error, otherwise the error code
	 */
	private int parseArguments(String[] argz)
	{
		StringBuilder sb = new StringBuilder();
		for(String s: argz)
			sb.append(s+" ");
		String s = sb.toString();
		String queries = s.substring(s.indexOf('\'')+1);
		String tails = queries.substring(queries.indexOf('\'')+1);
		queries = queries.substring(0, queries.indexOf('\''));
		
		keywords = divideIntoWords(queries);
		ArrayList<String> dividedTails = divideByBlank(tails);//dividedTails contains the precision requirement and the yahoo AppID
		
		if(keywords.size() == 0)
			return _ERROR_NUMBEROF_QUERY_WORDS_0_;
		if(dividedTails.size() != 2)
			return _ERROR_ARGUMENT_FORMAT_;
		
		precision = Double.parseDouble(dividedTails.get(0));
		if(precision < 0 || precision > 1)
			return _ERROR_ILLEGLE_PRECISION_VALUE;
		
		yahooID = dividedTails.get(1);
		//System.out.println("Parameters:");
		//System.out.println("Client key	="+yahooID);		
		//System.out.println("Query		="+queries);
		//System.out.println("Precision	="+precision);
		printlnToBoth("Parameters:");
		printlnToBoth("Client key	= "+yahooID);
		printlnToBoth("Query		= "+queries);
		printlnToBoth("Precision	= "+precision);
		return 0;
	}
	
	
	/**
	 * The function to get the query result and parse it into newArticles
	 * 
	 * @return 0 if no error, otherwise the error code
	 */
	private int getAndParseQuery()
	{
		StringBuilder sb = new StringBuilder(yahooURL_1);
		for(String s : keywords)
		{
			sb.append(s+"%20");
		}
		sb.append(yahooURL_2);
		sb.append(yahooID);
		sb.append(yahooURL_3);
		String query = sb.toString();
		//System.out.println("URL:" + query);
		printlnToBoth("URL:" + query);
		try {
			url = new URL(query);
			connection = url.openConnection();
			connection.setDoInput(true);
			inStream = connection.getInputStream();
		} catch (MalformedURLException e) {
			e.printStackTrace();
			return _ERROR_MALIC_URL_FORMAT_;
		} catch (IOException e) {
			e.printStackTrace();
			return _ERROR_IN_INTERNET_IO_; 
		}
		int NumRes = 0;
		NumRes = new XMLParser().parse(inStream, newArticles);
		if(NumRes == -1)
		{
			printlnToBoth("Error parsing the research result from Yahoo");
			return _ERROR_IN_PARSE_XML_RESULT_;
		}
		printlnToBoth("Total no of results : " + NumRes);
		return 0;
	}

	
	/**
	 * Print in Both Console and output file
	 * @param msg: the message to print
	 * @return
	 */
	private int printlnToBoth(String msg)
	{
		System.out.println(msg);
		try 
		{
			out.write(msg + "\r\n");
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		return 0;
	}
	private int printlnToBoth(ArrayList<String> arr)
	{
		System.out.println(arr);
		try 
		{
			out.write(arr + "\r\n");
		} catch (IOException e) {
			e.printStackTrace();
		}
		return 0;
	}
	
	
	/**
	 * The function dealing with user feedback, to display the query results
	 * and let user decide whether it's relevant or not;
	 * @param articles to display
	 * @return
	 */
	private int dispSearchResult(ArrayList<Article> articles){
		int relevantCnt = 0;
		int i = 0;
		printlnToBoth("Yahoo! Search Results: ");
		printlnToBoth("====================== ");
		for(Article article: articles){
			i++;
			printlnToBoth("Result "+i);
			printlnToBoth("[");
			printlnToBoth("URL: "+article.url);
			printlnToBoth("Title: "+article.title);
			printlnToBoth("Abstract: "+article.abs);
			printlnToBoth("]");
			printlnToBoth("");
			printlnToBoth("Relevant(Y/N)?");
			BufferedReader in=new BufferedReader(new InputStreamReader(System.in));

			try {
				String ans = in.readLine();
				out.write(ans);
				while(true)
				{
					ans = ans.toLowerCase();
					if(ans.equals("y") || ans.equals("yes") || ans.equals("\n"))
						{
							article.isRelevant = true;
							relevantCnt++; 
							break;
						}
					else if(ans.equals("n") || ans.equals("no"))
						{
							article.isRelevant = false;
							break;
						}
					else
					{
						printlnToBoth("Sorry, we do not understand your judgment, please re-type");
						printlnToBoth("Relevant(Y/N)?");
						ans = in.readLine( );
						out.write(ans);
					}
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
			
			printlnToBoth("");
		}
		
		printlnToBoth("====================== ");
		printlnToBoth("FEEDBACK SUMMARY");
		printlnToBoth("Query: "+keywords);
		printlnToBoth("Precision: "+relevantCnt/10.0);
		return relevantCnt;
	}

	
	/**
	 * Round up a vector to norm 1
	 * @param vector
	 */
	private void roundUp(double[] vector)
	{
		double sumOfSquare = 0.0;
		int i;
		for(i=0; i<vector.length; i++)
			sumOfSquare += vector[i] * vector[i];
		double squareRoot = Math.sqrt(sumOfSquare);
		for(i=0; i<vector.length; i++)
			vector[i] /= squareRoot;		
	};

	
	/**
	 * The major query expansion function, implementing the Rocchio algorithm:
	 * q_m = alpha * q_0 + beta * sum_{d_r}/|D_r| - gama * sum_{d_nr}/|D_nr|
	 * 
	 */
	private void queryExpansion()
	{
		for(Article article: newArticles)
			parseArticle(article);
		newArticles = new ArrayList<Article>();
		
		int nAllArticles = relevantArticles.size() + irrelevantArticles.size();
	
		for(Article article: relevantArticles)
		{
			article.vector = new double[currentWordId];
			for(String word: article.wordToCountmap.keySet())
			{				
				double tf = (double)article.wordToCountmap.get(word)/article.wordCount;
				double idf = Math.log((double)nAllArticles/wordToArticlemap.get(word).size())/Math.log(2.0);
				article.vector[wordIdMap.get(word)] = tf * idf;
			}
			roundUp(article.vector);
		}
		for(Article article: irrelevantArticles)
		{
			article.vector = new double[currentWordId];
			for(String word: article.wordToCountmap.keySet())
			{
				double tf = (double)article.wordToCountmap.get(word)/article.wordCount;
				double idf = Math.log((double)nAllArticles/wordToArticlemap.get(word).size())/Math.log(2.0);
				article.vector[wordIdMap.get(word)] = tf * idf;
			}
			roundUp(article.vector);
		}
		
		vector = new double[currentWordId];
		for(String word: keywords)
		{			
			double tf = (double)1/keywords.size();
			double idf = Math.log((double)nAllArticles/wordToArticlemap.get(word).size())/Math.log(2.0);
			vector[wordIdMap.get(word)] += tf * idf;
		}
		roundUp(vector);
		
		double[] newVector = new double[currentWordId];
		for(int i = 0 ; i < newVector.length; i++)
		{
			double fromRelevant = 0;
			double fromIrrelevant = 0;
			
			for(Article article: relevantArticles)
				fromRelevant += article.vector[i];
			fromRelevant/=relevantArticles.size();
			for(Article article: irrelevantArticles)
				fromIrrelevant += article.vector[i];
			fromIrrelevant/=irrelevantArticles.size();
			
			newVector[i] += alpha * vector[i] + beta * fromRelevant + gama * fromIrrelevant;				
		}
		roundUp(newVector);
		
		double first = -1;			// The largest value except for those're already keyword
		double second = -1;			// The second largest value except for those're already keyword
		int firstIndex = -1;
		int secondIndex = -1;
		int index = 0;
		
		HashSet<Integer> keywordsIndex = keywordsIndex();
		for(; index < newVector.length; index++)
		{
			double d = newVector[index];
		
			if(!keywordsIndex.contains(index) && !stopWords.contains(IdWordMap.get(index)))
			{
				if(d >= first)
				{
					second = first;
					secondIndex = firstIndex;
					first = d;
					firstIndex = index;
				}
				else if (d < first && d > second)
				{
					second = d;
					secondIndex = index;
				}
			}
		}
		ArrayList<String> toAdd = new ArrayList<String>();
		if(firstIndex != -1 && newVector[firstIndex] > 0)
		{
			keywords.add(IdWordMap.get(firstIndex));
			toAdd.add(IdWordMap.get(firstIndex));
		}
		if(secondIndex != -1 && newVector[secondIndex] > 0)
		{
			keywords.add(IdWordMap.get(secondIndex));		
			toAdd.add(IdWordMap.get(firstIndex));
		}
		if(toAdd.size() == 1)
			printlnToBoth("Augmenting by: "+toAdd.get(0)+"  "+IdWordMap.get(secondIndex));
		if(toAdd.size() == 2)
			printlnToBoth("Augmenting by: "+toAdd.get(0)+"  "+toAdd.get(1));

	}
	
	
	/**
	 * re-order the keywords acquired by query expansion
	 * only relevant articles are used to decide the ordering of the keywords
	 */	
	private void reorderKeywords()
	{
		double[][] relativePos = new double[keywords.size()][keywords.size()];		
		for(int i = 0; i < keywords.size();i++){
			for(int j =0; j < keywords.size();j++){
				//initialize to a very small number
				relativePos[i][j] = 0.000001;
			}
		}

		int[] totalOrder = new int[keywords.size()];
		for(int i = 0; i < keywords.size();i++)
			totalOrder[i] = 0;
		
		for(Article article: relevantArticles)
		{
			int[] posInArticle = new int[keywords.size()];
			for(int i =0; i < keywords.size(); i++)
			{
				String keyword = keywords.get(i);
				if(article.wordToPosMap.containsKey(keyword))
					posInArticle[i] = article.wordToPosMap.get(keyword);
				else
					posInArticle[i] = -1;
			}
			//now the position of all keywords in this article is stored in posInArticle
			//if keyword i doesn't appear in this article, posInArticle[i]=-1
			for(int i = 0; i < keywords.size();i++)
			{
				int frontKeywords = 0;
				if(posInArticle[i] != -1){//if this article contains keyword i 
					for(int j = 0; j < keywords.size();j++)
					{
						if(posInArticle[j] < posInArticle[i] && posInArticle[j] != -1)frontKeywords++;
					}
				}
				totalOrder[i] += (keywords.size()-frontKeywords);
			}			
			for(int i = 0; i < keywords.size();i++)
			{
				for(int j = 0; j < keywords.size();j++)
				{
					if(posInArticle[i] != -1 && posInArticle[j] != -1 && posInArticle[i] < posInArticle[j])
					{//if this article contains keyword i&j, and j appears after i
						relativePos[i][j] += 1.0/(posInArticle[j] - posInArticle[i]);
					}
				}
			}
		}
		
		ArrayList<String> keywordsCopy = new ArrayList<String>(keywords.size());
		
		int maxOrder = 0;
		int maxIndex = 0;
		for(int i = 0; i < keywords.size(); i++)
		{
			if(totalOrder[i] > maxOrder)
			{
				maxIndex = i;
				maxOrder = totalOrder[i];
			}
		}

		int preIndex = maxIndex;
		keywordsCopy.add(keywords.get(preIndex));
		int addIndex = 1;
		while(addIndex < keywords.size())
		{
			addIndex ++;
			double maxPosweight = 0;
			for(int i = 0; i < keywords.size(); i++)
			{
				if(relativePos[preIndex][i] > maxPosweight && !keywordsCopy.contains(keywords.get(i)))
				{
					maxIndex = i;
					maxPosweight = relativePos[preIndex][i];
				}						
			}
			preIndex = maxIndex;
			keywordsCopy.add(keywords.get(maxIndex));			
		}
		
		printlnToBoth("Before Ordering: ");
		System.out.println(keywords);
		
		keywords = keywordsCopy;
		
		printlnToBoth("After Ordering: ");
		System.out.println(keywords);
		
		printlnToBoth("New query: "+keywords);
	}
	
	
	
	
	/**
	 * Get the set of indices of current key words, used by queryExpansion() function
	 * @return the set of indices
	 */
	private HashSet<Integer> keywordsIndex()
	{
		
		HashSet<Integer> set = new HashSet<Integer>();
		for(String word: keywords)
		{
			set.add(wordIdMap.get(word));
			//System.out.format("----->See keywords: %s and index: %d\n", word, wordIdMap.get(word));
			//System.out.println();
		}
		return set;
	}
	
	
	/** 
	 * The function to divide a string into words,
	 * with empty words eliminated, all lower case, non digit or characters are deleted
	 * @param s: Input String
	 * @return: An arrayList of Strings divided by blanks
	 */
	private ArrayList<String> divideIntoWords(String s)
	{
		ArrayList<String> ret = new ArrayList<String>();
		String[] list = s.split(" ");
		for(String str: list)
		{
			str = str.replaceAll("[^a-zA-Z0-9]", "").toLowerCase();
			if(!str.equals(""))
				ret.add(str);
		}		
		return ret;
	}
	
	
	/** 
	 * The function to divide a string by blanks,
	 * @param s: Input String
	 * @return: An arrayList of Strings divided by blanks
	 */
	private ArrayList<String> divideByBlank(String s)
	{
		ArrayList<String> ret = new ArrayList<String>();
		String[] list = s.split(" ");
		for(String str: list)
		{
			if(!str.equals(""))
				ret.add(str);
		}		
		return ret;
	}
	
	
	
	/**
	 * @param article
	 * Parse one article, extract the title and abstract into words
	 * Update the wordToCountMap wordToPosMap field of the article
	 * Update the global wordIdMap, IdWordMap, and wordToArticalmap field
	 */	
	private void parseArticle(Article article)
	{		
		if(irrelevantArticles.contains(article) || relevantArticles.contains(article))
		{
			printlnToBoth("=========>This article has already been retrieved and parsed, return");
			return;
		}
		
		if(!article.isRelevant)
			irrelevantArticles.add(article);
		else
			relevantArticles.add(article);
				
		int pos = 0;
		String content = article.title;
		if(article.abs != null)
			content += " "+article.abs;
		ArrayList<String> words = divideIntoWords(content);
		for(String word: words)
		{			
			if(!wordIdMap.containsKey(word))
			{
				wordIdMap.put(word, currentWordId);
				IdWordMap.add(word);
				currentWordId++;
			}

			article.wordToCountmap.increment(word);
						
			if(!article.wordToPosMap.containsKey(word))
				article.wordToPosMap.put(word, pos++);
			
			wordToArticlemap.putElement(word, article);		
			
			article.wordCount++;
		}
	}
	
	
	
	
	public Client(String[] argz) {
		this.argz = argz;
		this.relevantArticles = new HashSet<Article>();
		this.irrelevantArticles = new HashSet<Article>();
		this.newArticles = new ArrayList<Article>();
		this.wordIdMap = new HashMap<String, Integer>();
		this.IdWordMap = new ArrayList<String>();
		this.wordToArticlemap = new SetMap<String, Article>();
		this.currentWordId = 0;
		this.file = new File("./transcript.txt");
		try {
			out = new FileWriter(file, true);
		} catch (IOException e) {
			e.printStackTrace();
		}
		this.initStopWords();
	}
	
	/**
	 * Initialize the set of stop words
	 */
	private void initStopWords()
	{
		this.stopWords = new HashSet<String>();
		stopWords.add("I");
		stopWords.add("a");
		stopWords.add("about");
		stopWords.add("an");
		stopWords.add("are");
		stopWords.add("as");
		stopWords.add("at");
		stopWords.add("be");
		stopWords.add("by");
		stopWords.add("com");
		stopWords.add("for");
		stopWords.add("from");
		stopWords.add("how");
		stopWords.add("in");
		stopWords.add("is");
		stopWords.add("it");
		stopWords.add("of");
		stopWords.add("on");
		stopWords.add("or");
		stopWords.add("that");
		stopWords.add("the");
		stopWords.add("this");
		stopWords.add("to");
		stopWords.add("was");
		stopWords.add("what");
		stopWords.add("when");
		stopWords.add("where");
		stopWords.add("who");
		stopWords.add("will");
		stopWords.add("with");
		stopWords.add("the");
		stopWords.add("www");
	}

	public static void main(String[] argz) throws IOException
	{
		client = new Client(argz);	
		
		if(client.parseArguments(argz) != 0)
			client.printlnToBoth("Error in arguments!");
		
		while(true)
		{
			client.getAndParseQuery();								
			double feedback = client.dispSearchResult(client.newArticles)/10.0;
			
			if(feedback == 0)
			{
				StringBuilder s = new StringBuilder();
				for(String str: client.keywords)
				{
					s.append(" ");
					s.append(str);
				}
				String keywordsstring = s.toString();
				
				client.printlnToBoth("There's no match articles for keywords:"+keywordsstring);
				client.out.close();
				return;

			}
			
			if(feedback < client.precision)
			{
				client.printlnToBoth("Still below the desired precision of " + client.precision);
				client.queryExpansion();
				client.reorderKeywords();				
			}
			else
				break;
		}
		client.printlnToBoth("Desired precision reached, done.");
		client.out.close();
	}
	
}
